#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Author;xiaobudian

from multiprocessing import Pool
import requests
from bs4 import BeautifulSoup
from fake_useragent import UserAgent
import pymongo
import os

ua = UserAgent()
headers1 = {'User-Agent': 'ua.ramdom'}
Mongo_Url = 'localhost'
Mongo_DB = 'Lianjia'
t_er_shou_fang = 'ershoufang'
t_sub_area = 'sub_area'
client = pymongo.MongoClient(Mongo_Url)
db = client[Mongo_DB]
base_url = "https://bj.lianjia.com"


def get_ershoufang_info(ershoufang_id):  # 分析详细url获取所需信息
    #    if db[Mongo_TABLE].find_one({"链家编号": ershoufang_id}) is not None:
    #        return None
    res = requests.get("https://bj.lianjia.com/ershoufang/" + ershoufang_id + ".html", 'lxml', headers=headers1)
    if res.status_code == 200:
        info = {}
        soup = BeautifulSoup(res.text, 'lxml')
        #        基本信息
        title = soup.select_one('.main')
        if title is None:
            return info
        info['标题'] = title.text
        total = soup.select_one('.total')
        if total is None:
            return info

        info['总价'] = total.text
        #        核心信息
        tag_tags = soup.find(class_='tags')
        if tag_tags is None:
            return info
        tag_as = tag_tags.find_all('a')
        tese = ''
        for a in tag_as:
            tese += a.text + ";"
        info['房源特色'] = tese
        info['每平方售价'] = soup.select('.unitPriceValue')[0].text
        info['建造时间'] = soup.select('.subInfo')[2].text
        info['小区名称'] = soup.select('.info')[0].text
        xiaoqu_info = soup.select_one('.info')

        if xiaoqu_info is None:
            return None
        xiaoqu_info_url = xiaoqu_info['href']

        info['小区编号'] = xiaoqu_info_url[8:-1]
        print(info['小区编号'])
        info['所在区域'] = soup.select('.info a')[0].text + ':' + soup.select('.info a')[1].text
        info['链家编号'] = ershoufang_id
        #       基本属性
        for i in soup.select('.base li'):
            i = str(i)
            if '</span>' in i or len(i) > 0:
                key, value = (i.split('</span>'))
                info[key[24:]] = value.rsplit('</li>')[0]
        #       交易属性
        for i in soup.select('.transaction li'):
            spans = i.find_all('span')
            info[spans[0].text] = spans[1].text.strip('\n').strip(' ').strip('\n')
        #
        return info


def update_to_MongoDB(info):  # update储存到MongoDB
    if info is None:
        return
    db[t_er_shou_fang].update({'链家编号': info['链家编号']}, {'$set': info})


def main(er_shou_fang_info):
    try:
        info = get_ershoufang_info(er_shou_fang_info['链家编号'])
        update_to_MongoDB(info)
    except:
        pass


if __name__ == '__main__':
    with Pool(os.cpu_count()) as pool:
        pool.map(main,
                 [er_shou_fang_info
                  for er_shou_fang_info in db[t_er_shou_fang].find().batch_size(50)
                  if '小区编号' not in er_shou_fang_info
                  or er_shou_fang_info['小区编号'] is None])
